setwd("~/accept_decline")

mean_impute = function(x)
{
	mu = mean(na.omit(x))
	x[is.na(x)] = mu
	x
}

df = readRDS("train.rds")
df2 = data.frame(lapply(df, mean_impute))[ ,-1]
a = sapply(df2, function(x) length(unique(x)))
keep_vars = names(a)[a>1]
df3 = df2[ , keep_vars]

quantilize = function(x)
{
	qntls = unique(quantile(x, seq(0, 1, .1)))
	print(length(qntls))
	if(length(qntls)>1)cut(x, qntls, 1:(length(qntls) - 1), include.lowest=T)
	else as.factor(rep(1, length(x)))
}



u = lapply(df3[,1:ncol(df3)], quantilize)
df4=data.frame(u, loss_ind=as.integer(df$loss>0))


require(Hmisc)
fnew = impute(df$f528 - df$f527)
df4$fnew = fnew

rm(df); rm(df2); rm(df3); gc()



f =  paste("loss_ind ~",paste(names(df4)[1:95], collapse="+"))
f = paste(f, "fnew",sep="+")

sapply(df4, function(x) length(unique(x)))
m = glm(f, data=df4, family=binomial)
summary(m) 


#f11?
#f2, f4, f7, f8, f9, f10, f13 (leak candidate), f16, f17, f19
#f25 (leak candidate), f26, f31, f32(leak candidate), f42, f45
#f46 f47, f49, f50, f53, f54, f55 (leak candidate), 57, 58, 62
#64, 65, 66, 67, 68, 69, 71, 72, 74, 75, 84, 94

#114, 132, 139, 142, 144, 149, 151, 153, 154, 161, 163, 173, 183, 181, 191, 193, 194, 201, 202, 203, 205
#209, 211, 213, 223, 224, 233, 238, 241, 251, 253, 261, 271, 278, 279, 280, 281, 282, 287, 288, 289, 292, 297,300,301
#307, 308, 309, 314, 315, 322, 324, 330, 333, 334, 340, 342,343,345, 346, 348,352,353, 355,368,370,376,377,383,395
#399, 402, 404, 413, 427, 442, 443, 457, 462, 463,468,471, 478
#513, 514, 515, 517, 518, 527, 528, 543, 573, 588, 589, 590, 597, 600
#611, 613, 615, 629, 630, 637, 666, 670, 671, 674,675, 681, 683, 684, 686, 687, 690,693, 705,707,709
#710, 712, 713, 714, 715, 717, 719, 720, 721, 722, 726, 727, 741, 755, 759, 772, 773, 774, 775


#latest model
m = glm(loss_ind~fnew + f2+f4+f13+f25+f67+f84+f94+f201+f211+f238+f340+f355+f376+
			     f404+f468+f471+f514+f515+f527+f528+f543+f588+f597+
			     f611+f615+f629+f637+f666+f670+f674+f675+f681+f683+f684+f686+f687+
				 f707+f715+f717+f721+f726+f727+f755+f772,
	data=df4, family=quasibinomial)
summary(m)

df5 = df4[,c("loss_ind", "f2","f4","f13","f25","f67","f84","f94","f201","f211","f238","f340","f355","f376",
  "f404","f468","f471","f514","f515","f527","f528","f543","f588","f597",
  "f611","f615","f629","f637","f666","f670","f674","f675","f681","f683","f684","f686","f687",
  "f693","f705","f707","f712","f715","f717","f721","f726","f727","f755","f772")]

m = glm(loss_ind~f2+f13+f25+f67+f84+f94+f201+f211+f238+f340+f355+f376+
			     f404+f468+f471+f514+f527+f528+f543+f588+f597+
			     f611+f615+f670+f674+f681+f683+f686+
				 f707+f712+f715+f717+f721+f726+f727+f755+f772,
	data=df_quantile, family=binomial)
	
m = glm(loss_ind~ tantrev+f2+f13+
			     f404+f468+f471+f543+f588+f597+
			     f611+f615+f670+f674+f681+f683+
				 f712+f715+f726+f727+f772,
	data=df_quantile, family=binomial)
summary(m)

require(MASS)
dropterm(m, test="Chisq")
plot(tapply(df_quantile$loss_ind,df_quantile$f772, mean))



m_fine = glm(loss_ind ~ f13 + f67 + f84 + f404 + f514+ f527 + f201 + f211 + f238, data=df5, family=binomial)
f674 + f471 + f468 +  +f528+f588, 

#lift chart
pred = predict(m_fine, type="response")
plot(quantile(pred, seq(0, 1, .001)), type="l")
a=cut(pred,quantile(pred, seq(0, 1, .001), include.lowest=T))
points(tapply(df5$loss_ind, a, mean))

#tree
rpart(loss_ind~f67+f84+f404+f514, data=df_bad, control=rpart.control(xval=5, cp=0.01))


sum(pred>.3)

df_bad=df5[pred>.2,]

m2 = glm(loss_ind ~ f674*f404, data=df5, family=binomial)

plot(tapply(df5$loss_ind, df5$f67, mean))




mean(df_bad$loss_ind[df_bad$f514< 82.928])


df_worst = subset(df_bad, 
 f3>=0.01879595
& f203< 9096.18
& f56>=0.704485
& f445< 44153.39
& f660< 1.435
& f168>=0.365
& f189>=0.77
& f122< 0.7)

mean(df_worst$loss_ind)


 f673< 6.8
& f358< 1.700785
& f674>=12.5
& f271< 20568.22
& f3>=0.01879595
& f203< 9096.18
& f56>=0.704485
& f445< 44153.39
& f660< 1.435
& f168>=0.365
& f189>=0.77
& f122< 0.7
